#! /usr/bin/php -q

<?php
/**
 *Copyright (c) 2012 copyright @ Xuehui YANG
 *website: http://www.bagualu.net
 *
 * @author Xuehui YANG southpasszju@gmail.com
 * @version 0.1 
 *
 * Licensed under The MIT License
 * Redistributions of files must retain the above copyright notice.
 *
*/
require ("simple_html_dom.php");
require ("dom_ex.php");
function is_utf8($str)
{
	return (utf8_encode(utf8_decode($str)) == $str); 
}
#$file="/home/xuyang/Downloads/books/www.tianyabook.com/sanguo/002.htm" ;
$file=$argv[1] ;
$str=file_get_contents($file);
//if(!is_utf8($str)){
//echo "convert to utf8 str\n";
//$str=mb_convert_encoding ( $str , "UTF-8", "CP936" ) ;
//}else{
//echo "the str is utf8 already\n";
//}

//convert to linux format
$str=preg_replace("/\x0d\x0a/","\x0a",$str);
$str=preg_replace("/<!--.*-->/","",$str);

$html=str_get_html($str);
//$html = file_get_html($file);


global $chapter_index;
global $chapter_title;
global $path;   //for the path with most chinese num , candidate for the tag to be parsed
global $path_idx;   //for the path idx
global $hzNum;  //used together with the above var, how many hz is in that tag
$leaf_tag_hz_num=array() ; //contain the leaf tag => hz num , in the order they appeared in the page
$total_hz_num=0;
$chapter_paragraphes=array();
$title_candidates=array();

//get the chapter_index and chapter_title
function toutf8($str)
{
	global $charset;
	$utf8= mb_convert_encoding ( $str , "UTF-8",$charset ) ;
	return $utf8;	
}
function get_title($html)
{
	$ret = $html->getElementByTagName("title");
	return  toutf8($ret->innertext) ;

}
function count_hz_num($str)
{
	$utf8= toutf8($str);
	//$utf8=$str;
	//$utf8= mb_convert_encoding ( $e->innertext , "UTF-8","CP936" ) ;
	preg_match_all('/[\x{4e00}-\x{9fff}]+/u', $utf8, $matches);
	$cont = join('', $matches[0]);
	$sz=mb_strlen($cont,"UTF-8");
	return $sz;
}
function isLeaf($e)
{
	$chs=$e->childNodes();
	$is=true;
	if(sizeof($chs)>0) { 
		foreach($chs as $ch){
			if( strcasecmp($ch->tag,"br") == 0 
			|| strcasecmp($ch->tag,"comment") == 0 
			|| count_hz_num($ch->innertext) == 0 ){
				
			}else{
				$is=false ;
			}
		}
	}
	return $is;	
}
function init($e,$_path,$idx)
{
	global $hzNum ; 
	global $path;
	global $leaf_tag_hz_num;
	global $total_hz_num;

	$text=$e->innertext ; 
	if($e->tag == "script") { return ;}  
	$cn=count_hz_num($text);
	if($cn > 0 && isLeaf($e)){
		$leaf_tag_hz_num[$_path] = $cn ; 
	}else if($cn > 0){
		//$leaf_tag_hz_num[$_path] = $cn ;
	}
	$utf8= toutf8($text) ;
	//$utf8=$text;
	if($cn > $hzNum ){
		$diff = $cn - $hzNum ; 
		$diff = $diff / $cn ; 
		$isFather = 0 ; 
		if(preg_match("/^$_path/",$path)){
			$isFather = 1 ; 
		}
		if($isFather && $diff < 0.3 ){
			echo "forget path " . $_path . "\n"; 
		}else{
		        $path = $_path ;
			$path_idx= $idx; 
		        $hzNum = $cn ; 
		        echo $path . " chinese count: " . $cn . ":\n" ;//. $utf8 . "\n";
		}
		
	}
	$arr=explode(" ",$_path);
	//echo $_path . " array size:" . sizeof($arr) . "\n";
	if(sizeof($arr) ==2){
		$total_hz_num = $cn ; 
	}
	return ; 	
}
function tranverse_html($html,$callback,$path="")
{
	$chs=$html->childNodes();
	$count=array();
	foreach ($chs as $ch){
		if(!isset($count[$ch->tag])) { $count[$ch->tag] = 0 ; }
		$cpath=$path . " " . $ch->tag . " " . $count[$ch->tag]; 
		tranverse_html($ch,$callback,$cpath);	
		//call_user_func_array($callback,array($ch,$cpath,$idx));
		call_user_func_array($callback,array($ch,$cpath,$count[$ch->tag]));
		$count[$ch->tag] ++ ; 
	}
}

$charset=$html->__get('charset');
$charset="CP936";
$title=get_title($html);
//print_r($html);
echo "title:" . $title . "\n" ;
//tag is $path, 
tranverse_html($html,"init");	
print_r($leaf_tag_hz_num);
echo "total hz:" . $total_hz_num . "\n"; 
$statistic_root_tag="";
foreach ($leaf_tag_hz_num as $tag=>$num){
	if( preg_match("/[a-z]$/",$tag) ){
		if($num / $total_hz_num > 0.8){
			$statistic_root_tag=$tag;	
			break;
		}
	}
}

//parse the tag and decide how to handle it.
echo "statistic root:" . $statistic_root_tag . "\n" ;
echo "root:" . $path . "\n" ;
if($statistic_root_tag != $path)
{
	echo "warning: two root does not match.\n";
}
$cont=get_node_by_tag($path);
if(!isset($cont)){
	echo "can not find the content tag\n";
}else{
	echo "get the content tag: " . $path .  " index " . $path_idx . "\n";
}

$tag=$cont;
//split the main hz struct and get the title candidate
//the following should be implemented in split.php and will be required
//after the process, get the title candidate list and main content block
// bookname dic is introduced and check if it is a book
//               if it is , process the main content
//               else , exit
require("split.php");

//foreach ($ret as $e)
//{
//	
//	//check_encoding($e->innertext);
//	//$cont=mb_convert_encoding ( $e->innertext , "UTF-8","CP936" ) ;
//	//echo "before conv to UTF-8:" . $e->innertext ; 
//	$cont=mb_convert_encoding ( $e->innertext , "UTF-8","CP936" ) ;
//	//echo "after conv to UTF-8:" . $cont ; 
//	//echo "disp done";
//	if($guoxue){
//		$cont = preg_replace("/<br>|<p>|<\/p>|&nbsp;| /", "", $cont);
//		array_push($chapter_paragraphes,$cont);
//	}else{
//		$cont=preg_replace("/br/","BR",$cont);
//		$cont=preg_replace("/(\x0a|\x0d|\x0a\x0d) +/","<BR>",$cont);
//		//preg_match_all('/[\x{e}-\x{9fff}]+/u', $cont, $matches);
//		//$cont = join('<BR>', $matches[0]);
//		echo "after replace oa:" . $cont . "\n";
//		$ps=explode("<BR>",$cont);
//		//$ps=explode("\n",$cont);
//		foreach( $ps as $p ) {
//			$p=preg_replace("/[A-Za-z_<>\-!0\/]/","",$p);;
//			$p=preg_replace("/^　+/u","",$p);;
//			echo "get a para" ;
//			echo $p ;
//			array_push($chapter_paragraphes,$p);
//		}
//	}
//	//$cont = preg_replace("/<br>|<p>|<\/p>/", "", $e->innertext);
//	//echo "get one para " . $cont ; 
//	//preg_match_all('/[\x{4e00}-\x{9fff}]+/u', $e->innertext, $matches);
//	//$cont = join('', $matches[0]);
//	//array_push($chapter_paragraphes,$e->innertext);
//	
//}
////print_r($ret);
////$cnt=0; 
////foreach ($ret as $e)
////{
////	foreach($e->find('p') as $p){
////		if ($cnt == 0){
////			foreach($p->find('span') as $t)
////			{
////				get_title($t->innertext);
////			}
////		}else{		
////			//array_push($chapter_paragraphes,mb_convert_encoding($p->innertext,'UTF-8','GB2312'));
////	yy		array_push($chapter_paragraphes,$p->innertext);
////		}
////		$cnt ++ ; 
////	}
////}
//
//
////$chapter["index"]=mb_convert_encoding($chapter_index,'UTF-8','GB2312');
////$chapter["title"]=mb_convert_encoding($chapter_title,'UTF-8','GB2312');
////$chapter["index"]=$chapter_index;
////$chapter["title"]=$chapter_title;
//$chapter["paragraphes"]=$chapter_paragraphes;
//
////print_r($chapter);  //gb2312
//
////save the chapter into a xml
////require("genxml.php");

?>
